package com.qianxun.spider;

import com.qianxun.spider.annote.SpiderBeanField;
import com.qianxun.spider.bean.SpiderBean;
import com.qianxun.spider.builder.SpiderBeanBuilder;
import com.qianxun.spider.config.SpiderBeanConfig;
import com.qianxun.spider.config.SpiderConfig;
import com.qianxun.spider.http.DefaultHttpProxyPool;
import com.qianxun.spider.http.HttpProxy;
import com.qianxun.spider.monitor.ConsoleSpiderMonitor;
import com.qianxun.spider.pipeline.ConsoleSpiderPipeline;
import lombok.Data;
import lombok.experimental.Accessors;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;

/**
 * @author: huwei
 * @date: 2020/6/24 9:04
 * @version: 1.0.0
 */
public class Demo {

    public static void main(String[] args) throws UnsupportedEncodingException {
//        //爬取指定内容，并转化为bean
//        spiderBeanClassNow();
//
//        //爬取指定内容，返回一个map对象
        spiderNow();

//        new ProxyIpSpiderDemo().startSpider();
    }

    private static void proxyTest(){
        //代理池配置
        DefaultHttpProxyPool proxyPool = new DefaultHttpProxyPool(3 ,true ,false);
        proxyPool.addLast(new HttpProxy("47.88.7.18" ,8088) ,15 * 60 * 60 * 1000);
        proxyPool.addLast(new HttpProxy("161.35.124.128" ,3128) ,15 * 60 * 60 * 1000);
        proxyPool.addLast(new HttpProxy("157.245.221.254" ,8080) ,15 * 60 * 60 * 1000);

        SpiderBean spiderBean = SpiderBeanBuilder.newly()
                .fieldBuilder().fieldName("ip").selector("title").valueKey("text").buildFieldAndAdd().build();

        SpiderBeanConfig spiderBeanConfig = SpiderBeanConfig.newly().urlTemplate("*")
                .pipeline(new ConsoleSpiderPipeline())
                .monitor(new ConsoleSpiderMonitor()).spiderBean(spiderBean);

        //爬虫相关配置，详情参见字段注释
        SpiderConfig spiderConfig = new SpiderConfig();
        spiderConfig.setConnTimeoutMillis(1000).setHttpProxyPool(proxyPool);

        Spider.newly().setUrl("http://202020.ip138.com/").config(spiderBeanConfig).config(spiderConfig.setDebug(false)).intervalMillis(1000).syncStart();
    }

    private static void spiderUrl(){
        SpiderBean spiderBean = SpiderBeanBuilder.newly()
                .fieldBuilder().fieldName("url").addTaskUrl().collection(true).selector("a").valueKey("href").buildFieldAndAdd().build();
        SpiderBeanConfig spiderBeanConfig = SpiderBeanConfig.newly().urlTemplate("*").pipeline(new ConsoleSpiderPipeline()).monitor(new ConsoleSpiderMonitor()).spiderBean(spiderBean);
        Spider.newly().setUrl("http://www.530p.com").config(spiderBeanConfig).intervalMillis(1000).syncStart();
    }


    /**
     * 同步爬取
     * 等待并获得爬取数据
     */
    private static void spiderNow() throws UnsupportedEncodingException {
        //目标地址
        String url = "http://www.530p.com/s/" + URLEncoder.encode("明天下" ,"UTF-8");
        //动态组成爬取页面数据内容，配置字段含义可参考com.qianxun.spider.bean.SpiderBean 和 com.qianxun.spider.bean.SpiderBeanField
        SpiderBean spiderBean = SpiderBeanBuilder.newly()
                .fieldBuilder().fieldName("bookName").collection(true).selector("body > div:nth-child(3) > div.conter > ul > li.conter1 > a").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("bookHref").collection(true).selector("body > div:nth-child(3) > div.conter > ul > li.conter1 > a").valueKey("href").buildFieldAndAdd()
                .fieldBuilder().fieldName("lastChapterName").collection(true).selector("body > div:nth-child(3) > div.conter > ul > li.conter2 > a").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("lastChapterHref").collection(true).selector("body > div:nth-child(3) > div.conter > ul > li.conter2 > a").valueKey("href").buildFieldAndAdd()
                .fieldBuilder().fieldName("author").collection(true).selector("body > div:nth-child(3) > div.conter > ul > li.conter4").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("lastUpdateTime").collection(true).selector("body > div:nth-child(3) > div.conter > ul > li.conter3").valueKey("text").buildFieldAndAdd()
                .build();

        //爬取目标数据
        Object obj = Spider.get(spiderBean ,url);
        System.out.println(obj);
    }

    /**
     * 异步爬取
     * 通过自定义的SpiderMonitor监管数据爬取情况
     * 通过自定义的SpiderPipeline处理爬取并解析后的数据
     */
    private static void syncSpider() {
        //爬取内容动态组装，少复杂可忽略；
        SpiderBean spiderBean = SpiderBeanBuilder.newly().beanClass(Book.class)
                .fieldBuilder().fieldName("name").selector("body > div:nth-child(3) > div.tna > a").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("author").selector("body > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(3)").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("lastChapterUpdateTime").selector("body > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(7)").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("lastChapterName").selector("body > div:nth-child(3) > table > tbody > tr:nth-child(2) > td > a").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("lastChapterHref").addTaskUrl().selector("body > div:nth-child(3) > table > tbody > tr:nth-child(2) > td > a").valueKey("href").buildFieldAndAdd()
                .fieldBuilder().fieldName("description").selector("body > div:nth-child(3) > table > tbody > tr:nth-child(4) > td").valueKey("text").buildFieldAndAdd()
                .fieldBuilder().fieldName("cover").addTaskUrl().selector("body > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(1) > img").valueKey("src").buildFieldAndAdd()
                .fieldBuilder().fieldName("chapterList").collection(true).spiderBean(SpiderBeanBuilder.newly().beanClass(Chapter.class)
                        .fieldBuilder().fieldName("chapterName").collection(true).selector("body > div:nth-child(3) > div.conter > div > a").valueKey("text").buildFieldAndAdd()
                        .fieldBuilder().fieldName("chapterHref").collection(true).selector("body > div:nth-child(3) > div.conter > div > a").valueKey("href").buildFieldAndAdd()
                        .build()).buildFieldAndAdd()
                .build();

        //配置SpiderBean对应的解析url地址模板、爬取数据监听、解析数据监听、以及爬取数据管道输出
        //SpiderBeanConfig.urlTemplate ：设定SpiderBean解析的页面模板url地址，即爬取的页面URl地址与该地址匹配时，才运用该spiderBean进行内容解析；详情查看该字段的注释
        //ConsoleSpiderMonitor ： 数据监听
        //ConsoleSpiderPipeline : 爬取数据输出管道
        SpiderBeanConfig spiderBeanConfig = SpiderBeanConfig.newly()
                .spiderBean(spiderBean).urlTemplate("http://www.530p.com/qihuan/shinue-9534/")
                .pipeline(new ConsoleSpiderPipeline()).monitor(new ConsoleSpiderMonitor());

        //异步爬取：setUrl 配置起始爬取地址
        Spider.newly().debug(true).config(spiderBeanConfig).setUrl("http://www.530p.com/qihuan/shinue-9534/").syncStart();
    }

    private static void spiderBeanClassNow(){
        Object obj = Spider.get(SpiderBeanBuilder.newly().buildClass(Book.class) ,"http://www.530p.com/qihuan/shinue-9534/");
        System.out.println(obj);
    }

    @Data
    @com.qianxun.spider.annote.SpiderBean
    public static class Chapter {
        @SpiderBeanField(collection = true ,selector = "body > div:nth-child(3) > div.conter > div > a" ,valueKey = "text")
        private String chapterName;
        @SpiderBeanField(collection = true ,selector = "body > div:nth-child(3) > div.conter > div > a" ,valueKey = "href")
        private String chapterHref;
        public Chapter() {
        }
    }

    @Data
    @Accessors(chain = true)
    @com.qianxun.spider.annote.SpiderBean
    public static class Book {
        @SpiderBeanField(selector = "body > div:nth-child(3) > div.tna > a" ,valueKey = "text")
        private String name;
        @SpiderBeanField(selector = "body > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(3)" ,valueKey = "text")
        private String author;
        @SpiderBeanField(selector = "body > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(1) > img" ,valueKey = "src")
        private String cover;
        @SpiderBeanField(selector = "body > div:nth-child(3) > table > tbody > tr:nth-child(4) > td" ,valueKey = "text")
        private String description;
        @SpiderBeanField(selector = "body > div:nth-child(3) > table > tbody > tr:nth-child(2) > td > a" ,valueKey = "text")
        private String lastChapterName;
        @SpiderBeanField(selector = "body > div:nth-child(3) > table > tbody > tr:nth-child(2) > td > a" ,valueKey = "text")
        private String lastChapterHref;
        @SpiderBeanField(selector = "body > div:nth-child(3) > table > tbody > tr:nth-child(1) > td:nth-child(7)" ,valueKey = "text")
        private String lastChapterUpdateTime;
        @SpiderBeanField(spiderBeanClass = Chapter.class ,collection = true)
        private List<Chapter> chapterList;
    }
}
